package com.hymake.processor;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.hymake.model.LiuyanShen;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.model.HttpRequestBody;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.JsonPathSelector;
import us.codecraft.webmagic.utils.HttpConstant;

/**
 * 海迈房地产海南留言
 * 
 */
@Component
public class LiuyanShenProcessor implements PageProcessor {
	
    @Autowired
    private Pipeline liuyanShenDaoPipeline;
    
    public static final String URL_LIST = "http://liuyan.people.com.cn/threads/queryThreadsList";

	private Site site = Site.me();

    public void process(Page page) {
		if (page.getUrl().regex(URL_LIST).match()) {
			List<String> ids = new JsonPathSelector("$.responseData[*].tid").selectList(page.getRawText());
			
			for (String id : ids) {
				page.addTargetRequest("http://liuyan.people.com.cn/threads/content?tid=" + id);
			}
			Request request = setRequest(Integer.parseInt(ids.get(ids.size()-1)));
			page.addTargetRequest(request);
		} else {
			LiuyanShen liuyan = new LiuyanShen();
			liuyan.setTitle(page.getHtml().xpath("//div[@class='liuyan_box03 w1200 clearfix']/h2/b/text()").toString());
			liuyan.setContent(page.getHtml().xpath("//p[@class='zoom']/text()").toString());
			liuyan.setPublishTime((page.getHtml().xpath("//h3/span/text()").toString()));
			
			page.putField("repo", liuyan);
		}
     	
//       

    }
    
    public Site getSite() {
		return site.addHeader("Accept", "application/json, text/javascript, */*; q=0.01")
				.addHeader("Accept-Encoding", "gzip, deflate")
				.addHeader("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2")
				.addHeader("Cache-Control", "max-age=0")
				.addHeader("Connection", "keep-alive")
				.addHeader("Content-Type", "application/x-www-form-urlencoded")
				.addHeader("Cookie","aliyungf_tc=AQAAAPF/xiZrwgoAm/s5O+JZwvjy91UG; JSESSIONID=B4301E737F6237A965AFFFC621364488; wdcid=15c0b4ff7b61aa0a; wdlast=1532413694; wdses=201563e453c0d7f1")
				.addHeader("Host", "liuyan.people.com.cn")
				.addHeader("Referer", "http://liuyan.people.com.cn/threads/list?fid=579")
				.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; W…) Gecko/20100101 Firefox/61.0")
				.addHeader("X-Requested-With", "XMLHttpRequest");
    }
	
	public void crawl(PageProcessor crawler) {
		Spider.create(crawler)
				.addPipeline(liuyanShenDaoPipeline)//
				.thread(1)
				.addRequest(setRequest(0))
				.run();
	}

	private Request setRequest(int lastItem) {
		Request request = new Request("http://liuyan.people.com.cn/threads/queryThreadsList");
		request.setMethod(HttpConstant.Method.POST);
		Map<String, Object> params = new HashMap<String, Object>();
		params.put("fid", 580);
		params.put("lastItem", lastItem); // 0 5259529
		request.setRequestBody(HttpRequestBody.form(params , "utf-8"));
		return request;
	}

}
